import urllib2, os
from BeautifulSoup import *
from urlparse import urljoin


path = './../trec_data_set'
trec_list = os.listdir(path)

dir_count = 0
for i in trec_list:
	dir_count = dir_count + 1
	path_trec = path + '/' + i
	print "#####################################################################\t" + str(dir_count)
	print "#####################################################################"
	print 'Opening Directory----->' + path_trec
	B_list = os.listdir(path_trec)

	file_count = 0
	for j in B_list:
		file_count = file_count + 1
		path_B = path_trec + '/' + j
		print "#####################################################################\t" + str(dir_count) + '\t' + str(file_count)
		print 'Opening file------->' + path_B

		fp = open(path_B,"r")
		data = fp.read()
		data = unicode(data, errors = 'ignore')

		soup  = BeautifulSoup(data)
		docs = soup.contents

		for i in range(0,len(docs),2):
			list_links = docs[i]('a')
			docno = str(docs[i].docno)
			dochdr = str(docs[i].dochdr)

			t = (dochdr[8:]).split(' ')
			docurl = t[0]
			docid = docno[7:-8]
			print docurl
			
			out = 0
			for link in list_links:
				if ('href' in dict(link.attrs)):
					url = urljoin(docurl,link['href'])
					if url.find("'") != -1: 
						continue
					url = url.split('#')[0]
					if url[0:4] == 'http':
						if url[0] == '"':
							url = url[1:]
						if url[-1] == '"':
							url = url[:-1]
						try:
							print "Indexing link %s" % url
						except:
							tempUrl = repr(url)
							tempUrl = str(tempUrl)
							temp = tempUrl.split('>')
							url = temp[0][2:]
							print "Indexing link %s" % url
						out = out + 1
			print '\n'
		fp.close()
